#!/bin/bash
# --------------------
# Setup
# --------------------

nodes=1
mailtype="ALL"

# Manual settings for years
start_year=1986
end_year=2021

# for this version second argument is job name
job=${2}

# if user has shell profile setup
if [ -f ~/shell_profile.sh ]; then

    # run profile
    source ~/shell_profile.sh

    U=${whoami}
    echo "User: "${whoami}

    # basic variables
    system_part="<SLURM_PARTITION>"

    # morningstar specific paths
    code_path="<CODE_PATH>"
    erroroutput_path="$project_path/erroroutput"
    morningstar_data_path="$project_path"
    read_xml_data_path="$project_path/raw/morningstar"

    # make error output folder if it doesn't exist
    mkdir -p "${erroroutput_path}"
    
    # job parameters
	partition_externalid_postbloomberg="$system_part"
    time_externalid_postbloomberg="0-2:00:00"
    ntasks_externalid_postbloomberg=8
    mem_externalid_postbloomberg="500000"

	partition_Refine_Cusip_Fill_Isin_1="$system_part"
    time_Refine_Cusip_Fill_Isin_1="0-5:00:00"
    ntasks_Refine_Cusip_Fill_Isin_1=28
    mem_Refine_Cusip_Fill_Isin_1="41500"

	partition_Refine_Cusip_Fill_Isin_2="$system_part"
    time_Refine_Cusip_Fill_Isin_2="0-5:00:00"
    ntasks_Refine_Cusip_Fill_Isin_2=8
    mem_Refine_Cusip_Fill_Isin_2="350000"

	partition_externalid_make="$system_part"
    time_externalid_make="0-40:00:00"
    ntasks_externalid_make=8
    mem_externalid_make="500000"

	partition_Refine_Extid_Merge_1="$system_part"
    time_Refine_Extid_Merge_1="0-8:00:00"
    ntasks_Refine_Extid_Merge_1=28
    mem_Refine_Extid_Merge_1="41500"

	partition_Refine_Extid_Merge_2="$system_part"
    time_Refine_Extid_Merge_2="0-8:00:00"
    ntasks_Refine_Extid_Merge_2=8
    mem_Refine_Extid_Merge_2="350000"

	partition_Internal_Currency="$system_part"
    time_Internal_Currency="0-20:00:00"
    ntasks_Internal_Currency=8
    mem_Internal_Currency="500000"

	partition_Refine_Cusip_Merge_1="$system_part"
    time_Refine_Cusip_Merge_1="0-8:00:00"
    ntasks_Refine_Cusip_Merge_1=8
    mem_Refine_Cusip_Merge_1="41500"

	partition_Refine_Cusip_Merge_2="$system_part"
    time_Refine_Cusip_Merge_2="0-8:00:00"
    ntasks_Refine_Cusip_Merge_2=28
    mem_Refine_Cusip_Merge_2="600000"

	partition_Internal_Class="$system_part"
    time_Internal_Class="0-8:00:00"
    ntasks_Internal_Class=8
    mem_Internal_Class="350000"

	partition_Manual_Corrections="$system_part"
    time_Manual_Corrections="0-8:00:00"
    ntasks_Manual_Corrections=8
    mem_Manual_Corrections="600000"

	partition_Create_Final_Files_1="$system_part"
    time_Create_Final_Files_1="0-8:00:00"
    ntasks_Create_Final_Files_1=28
    mem_Create_Final_Files_1="41500"

	partition_Create_Final_Files_2="$system_part"
    time_Create_Final_Files_2="0-8:00:00"
    ntasks_Create_Final_Files_2=8
    mem_Create_Final_Files_2="350000"

	partition_Unwind_MF_Prepare="$system_part"
    time_Unwind_MF_Prepare="0-05:00:00"
    ntasks_Unwind_MF_Prepare=4
    mem_Unwind_MF_Prepare="300000"

    partition_Unwind_MF_Positions_Unravel="$system_part"
    time_Unwind_MF_Positions_Unravel="0-10:00:00"
    ntasks_Unwind_MF_Positions_Unravel=4
    mem_Unwind_MF_Positions_Unravel="300000"

    partition_Unwind_MF_Positions_Consolidate="$system_part"
    time_Unwind_MF_Positions_Consolidate="0-2:00:00"
    ntasks_Unwind_MF_Positions_Consolidate=4
    mem_Unwind_MF_Positions_Consolidate="300000"

    partition_Unwind_MF_Positions_ReGenHD_1_Small="$system_part"
    time_Unwind_MF_Positions_ReGenHD_1_Small="0-10:00:00"
    ntasks_Unwind_MF_Positions_ReGenHD_1_Small=4
    mem_Unwind_MF_Positions_ReGenHD_1_Small="41500"

    partition_Unwind_MF_Positions_ReGenHD_1_Large="$system_part"
    time_Unwind_MF_Positions_ReGenHD_1_Large="0-10:00:00"
    ntasks_Unwind_MF_Positions_ReGenHD_1_Large=4
    mem_Unwind_MF_Positions_ReGenHD_1_Large="300000"

    partition_Unwind_MF_Positions_ReGenHD_2_Small="$system_part"
    time_Unwind_MF_Positions_ReGenHD_2_Small="0-10:00:00"
    ntasks_Unwind_MF_Positions_ReGenHD_2_Small=4
    mem_Unwind_MF_Positions_ReGenHD_2_Small="41500"

    partition_Unwind_MF_Positions_ReGenHD_2_Large="$system_part"
    time_Unwind_MF_Positions_ReGenHD_2_Large="0-18:00:00"
    ntasks_Unwind_MF_Positions_ReGenHD_2_Large=4
    mem_Unwind_MF_Positions_ReGenHD_2_Large="300000"

    partition_Unwind_MF_Positions_Aggregate="$system_part"
    time_Unwind_MF_Positions_Aggregate="0-10:00:00"
    ntasks_Unwind_MF_Positions_Aggregate=4
    mem_Unwind_MF_Positions_Aggregate="300000"

    partition_Clean_Temp="$system_part"
    time_Clean_Temp="0-00:10:00"
    ntasks_Clean_Temp=1
    mem_Clean_Temp="25000"

    partition_Reshape_Cut_Groups="$system_part"
    time_Reshape_Cut_Groups="0-2:00:00"
    ntasks_Reshape_Cut_Groups=12
    mem_Reshape_Cut_Groups="175000"

    partition_Split_Annual_Files="$system_part"
    time_Split_Annual_Files="0-20:00:00"
    ntasks_Split_Annual_Files=6
    mem_Split_Annual_Files="350000"

    partition_Reappend_Split_Annuals="$system_part"
    time_Reappend_Split_Annuals="0-1:30:00"
    ntasks_Reappend_Split_Annuals=4
    mem_Reappend_Split_Annuals="60000"

    partition_Panelify_MPID_Splits="$system_part"
    time_Panelify_MPID_Splits="0-06:00:00"
    ntasks_Panelify_MPID_Splits=6
    mem_Panelify_MPID_Splits="190000"

    partition_Panelify_MPID_Splits_Covariates="$system_part"
    time_Panelify_MPID_Splits_Covariates="0-06:00:00"
    ntasks_Panelify_MPID_Splits_Covariates=6
    mem_Panelify_MPID_Splits_Covariates="190000"

    partition_Append_HD_Panels="$system_part"
    time_Append_HD_Panels="0-10:00:00"
    ntasks_Append_HD_Panels=12
    mem_Append_HD_Panels="310000"

    partition_Find_Constant_Panels="$system_part"
    time_Find_Constant_Panels="0-4:00:00"
    ntasks_Find_Constant_Panels=8
    mem_Find_Constant_Panels="50000"

    partition_Collapse_Panels="$system_part"
    time_Collapse_Panels="0-8:00:00"
    ntasks_Collapse_Panels=12
    mem_Collapse_Panels="220000"

else

    echo "Please set up ~/shell_profile.sh file. Exiting."
    exitcd

fi

# sanity-checking that project_path variables exist
if [[ -z "$project_path" ]]; then
    echo "Empty project_path variable: Exiting..."
    exit 1
fi

# Export path variables
export code_path
export project_path

############################################################################################################################
# DEFINE ARRAYS FOR JOBS UP TO UNWIND STEPS
############################################################################################################################

# Hard coded year setting based on files sizes and data structure changes.
# These variables do not affect which years are run by the build.
switch_year=2005

# Set variable for whether to process large HoldingDetail files (those from 2003 to 2016)
if ((end_year > switch_year)); then
    run_large=true
else
    run_large=false
fi
# Set variable for whether to process small HoldingDetail files (those from 1986 to 2002)
if ((start_year <= switch_year)); then
    run_small=true
else
    run_small=false
fi

# Define small array for jobs
small_end=$((switch_year > end_year ? end_year : switch_year))
array_small="${start_year}-${small_end}"
# Define large array for jobs
large_start=$((switch_year+1 > start_year ? switch_year+1 : start_year))
array_large="${large_start}-${end_year}"


############################################################################################################################
# DEFINE ARRAYS FOR JOBS IN UNWIND STEPS
############################################################################################################################

# count total number of years
n_years=$(($end_year - $start_year + 1))
# count of years in first split interval (1986-2005)
n_switch_years1=$(($switch_year - $start_year + 1))
# count of years in second split interval (2006-2020)
n_switch_years2=$(($end_year - $switch_year))

# count number of NonUS, 1986-2005
n_NonUS1=$(( $n_switch_years1 * 2 - 1))
# count start of NonUS, 2006-2020
n_NonUS2=$(($n_NonUS1 + 1))
# count number of halfs years, 1986-2020/end of NonUS, 1986-2020 (since we manually check 1986 h1)
n_halfs=$(( ($n_years) * 2 - 1))

#71 start of US, 1986-2005
n_US0=$(($n_halfs + 2))
#109 end of US, 1986-2005
n_US1=$(($n_US0 + $n_NonUS1 - 1))
#110 start of US, 2006-2020
n_US2=$(($n_US1 + 1))
#139 end of US, 2006-2020
n_US3=$(($n_halfs * 2 + 1))


# Export path variables
echo "Code path = "${code_path}
echo "Data path = "${morningstar_data_path}
export code_path
export morningstar_data_path


############################################################################################################################
# SUBMIT FRIST STEPS
############################################################################################################################

# this version runs a single jobs
case ${job} in
"first_batches" | "all" | "pre_unwind")

# Externalid (Post-Bloomberg Stage)
### This stage is used to process the externalid part after the manual step using the OpenFIGI API and Bloomberg
### It uses OpenFIGI as well as the corresponding raw data pulled from the Bloomberg terminal to match to data in the Morningstar to create a link between externalid_mns and cusip/isin in master files
externalid_postbloomberg_ID=`sbatch \
         --partition=${partition_externalid_postbloomberg} --time=${time_externalid_postbloomberg} \
         --nodes=${nodes} --ntasks=${ntasks_externalid_postbloomberg} --job-name=externalid_postbloomberg \
		 --output="${erroroutput_path}/externalid_postbloomberg-%A_%a.out" --error="${erroroutput_path}/externalid_postbloomberg-%A_%a.err" \
		 --mail-user=${mailuser} --mail-type=${mailtype} --requeue --mem=${mem_externalid_postbloomberg} \
         "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} externalid_postbloomberg ${start_year} ${end_year} | awk '{print $NF}'`
echo "Submitted externalid_postbloomberg Job: "${externalid_postbloomberg_ID}
sleep 1

# Refine Cusip Fill Isin
#### This step performs a series of data cleaning steps that improve the quality of security metadata (i.e. cusip and isin) by merging in information from the CGS security master files and the OpenFIGI/Bloomberg data pull.
depend_externalid_collect_extid_master=""
if [ "$run_small" = true ] ; then
    Refine_Cusip_Fill_Isin_1_ID=`sbatch \
             --partition=${partition_Refine_Cusip_Fill_Isin_1} --time=${time_Refine_Cusip_Fill_Isin_1} \
             --nodes=${nodes} --ntasks=${ntasks_Refine_Cusip_Fill_Isin_1} --job-name=Refine_Cusip_Fill_Isin_1 \
             --output="${erroroutput_path}/Refine_Cusip_Fill_Isin_1-%A_%a.out" --error="${erroroutput_path}/Refine_Cusip_Fill_Isin_1-%A_%a.err" \
             --array=${array_small} \
             --mail-user=${mailuser} --mail-type=${mailtype} --requeue --mem=${mem_Refine_Cusip_Fill_Isin_1} \
             --depend=afterok:${externalid_postbloomberg_ID} \
             "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Refine_Cusip_Fill_Isin ${start_year} ${end_year} | awk '{print $NF}'`
    echo "Submitted Refine_Cusip_Fill_Isin_1 Job: "${Refine_Cusip_Fill_Isin_1_ID}
    sleep 1

    depend_externalid_collect_extid_master=":${Refine_Cusip_Fill_Isin_1_ID}"
fi

if [ "$run_large" = true ] ; then
    Refine_Cusip_Fill_Isin_2_ID=`sbatch \
             --partition=${partition_Refine_Cusip_Fill_Isin_2} --time=${time_Refine_Cusip_Fill_Isin_2} \
             --nodes=${nodes} --ntasks=${ntasks_Refine_Cusip_Fill_Isin_2} --job-name=Refine_Cusip_Fill_Isin_2 \
             --output="${erroroutput_path}/Refine_Cusip_Fill_Isin_2-%A_%a.out" --error="${erroroutput_path}/Refine_Cusip_Fill_Isin_2-%A_%a.err" \
             --array=${array_large} \
             --mail-user=${mailuser} --mail-type=${mailtype} --requeue --mem=${mem_Refine_Cusip_Fill_Isin_2} \
             --depend=afterok:${externalid_postbloomberg_ID} \
             "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Refine_Cusip_Fill_Isin ${start_year} ${end_year} | awk '{print $NF}'`
    echo "Submitted Refine_Cusip_Fill_Isin_2 Job: "${Refine_Cusip_Fill_Isin_2_ID}
    sleep 1

    depend_externalid_collect_extid_master="${depend_externalid_collect_extid_master}:${Refine_Cusip_Fill_Isin_2_ID}"
fi


############################################################################################################################
# SUBMIT SECOND STEPS
############################################################################################################################

# job separator
;;&
"second_batches" | "all" | "pre_unwind")

# set up dependcy if necessary 
# Set variable for whether to process small HoldingDetail files (those from 1986 to 2002)
if [ "$job" = "second_batches" ] ; then
    depend=""
else
    depend="--depend=afterok${depend_externalid_collect_extid_master}"
fi

# Externalid Make
#### This step generates an internal flatfile which has all security-level details for each externalid in the Morningstar holdings data.
externalid_collect_extid_master_ID=`sbatch \
         --partition=${partition_externalid_make} --time=${time_externalid_make} \
         --nodes=${nodes} --ntasks=${ntasks_externalid_make} --job-name=externalid_collect_extid_master \
		 --output="${erroroutput_path}/externalid_collect_extid_master-%A_%a.out" --error="${erroroutput_path}/externalid_collect_extid_master-%A_%a.err" \
		 --mail-user=${mailuser} --mail-type=${mailtype} --requeue --mem=${mem_externalid_make} \
         ${depend} \
         "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} externalid_collect_extid_master ${start_year} ${end_year} | awk '{print $NF}'`
echo "Submitted externalid_collect_extid_master Job: "${externalid_collect_extid_master_ID}
sleep 1

make_externalid_master_ID=`sbatch \
         --partition=${partition_externalid_make} --time=${time_externalid_make} \
         --nodes=${nodes} --ntasks=${ntasks_externalid_make} --job-name=make_externalid_master \
		 --output="${erroroutput_path}/make_externalid_master-%A_%a.out" --error="${erroroutput_path}/make_externalid_master-%A_%a.err" \
		 --mail-user=${mailuser} --mail-type=${mailtype} --requeue --mem=${mem_externalid_make} \
		 ${depend} \
         "${code_path}/morningstar/externalid/R_Controller.sh" ${U} make_externalid_master | awk '{print $NF}'`
echo "Submitted make_externalid_master Job: "${make_externalid_master_ID}
sleep 1

externalid_make_extid_csvtodta_ID=`sbatch \
         --partition=${partition_externalid_make} --time=${time_externalid_make} \
         --nodes=${nodes} --ntasks=${ntasks_externalid_make} --job-name=externalid_make_extid_csvtodta \
		 --output="${erroroutput_path}/externalid_make_extid_csvtodta-%A_%a.out" --error="${erroroutput_path}/externalid_make_extid_csvtodta-%A_%a.err" \
		 --mail-user=${mailuser} --mail-type=${mailtype} --requeue --mem=${mem_externalid_make} \
		 --depend=afterok:${make_externalid_master_ID} \
         "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} externalid_make_extid_csvtodta ${start_year} ${end_year} | awk '{print $NF}'`
echo "Submitted externalid_make_extid_csvtodta Job: "${externalid_make_extid_csvtodta_ID}
sleep 1

# Refine Extid Merge
#### This step improves the quality of security information included in the Morningstar data by merging information from the internally-generated externalid master file into the holdings data

depend_Refine_Extid_Merge=""
if [ "$run_small" = true ] ; then
    Refine_Extid_Merge_1_ID=`sbatch \
             --partition=${partition_Refine_Extid_Merge_1} --time=${time_Refine_Extid_Merge_1} \
             --nodes=${nodes} --ntasks=${ntasks_Refine_Extid_Merge_1} --job-name=Refine_Extid_Merge_1 \
             --output="${erroroutput_path}/Refine_Extid_Merge_1-%A_%a.out" --error="${erroroutput_path}/Refine_Extid_Merge_1-%A_%a.err" \
             --array=${array_small} \
             --mail-user=${mailuser} --mail-type=${mailtype} --requeue --mem=${mem_Refine_Extid_Merge_1} \
             --depend=afterok:${externalid_make_extid_csvtodta_ID} \
             "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Refine_Extid_Merge ${start_year} ${end_year} | awk '{print $NF}'`
    echo "Submitted Refine_Extid_Merge_1 Job: "${Refine_Extid_Merge_1_ID}
    sleep 1

    depend_Refine_Extid_Merge=":${Refine_Extid_Merge_1_ID}"
fi
if [ "$run_large" = true ] ; then
    Refine_Extid_Merge_2_ID=`sbatch \
             --partition=${partition_Refine_Extid_Merge_2} --time=${time_Refine_Extid_Merge_2} \
             --nodes=${nodes} --ntasks=${ntasks_Refine_Extid_Merge_2} --job-name=Refine_Extid_Merge_2 \
             --output="${erroroutput_path}/Refine_Extid_Merge_2-%A_%a.out" --error="${erroroutput_path}/Refine_Extid_Merge_2-%A_%a.err" \
             --array=${array_large} \
             --mail-user=${mailuser} --mail-type=${mailtype} --requeue --mem=${mem_Refine_Extid_Merge_2} \
             --depend=afterok:${externalid_make_extid_csvtodta_ID} \
             "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Refine_Extid_Merge ${start_year} ${end_year} | awk '{print $NF}'`
    echo "Submitted Refine_Extid_Merge_2 Job: "${Refine_Extid_Merge_2_ID}
    sleep 1

    depend_Refine_Extid_Merge="${depend_Refine_Extid_Merge}:${Refine_Extid_Merge_2_ID}"
fi

# Internal Currency
#### This step constructs a data with a modal currency assignments for each fund in the Morningstar data.
Internal_Currency_ID=`sbatch \
         --partition=${partition_Internal_Currency} --time=${time_Internal_Currency} \
         --nodes=${nodes} --ntasks=${ntasks_Internal_Currency} --job-name=Internal_Currency \
		 --output="${erroroutput_path}/Internal_Currency-%A_%a.out" --error="${erroroutput_path}/Internal_Currency-%A_%a.err" \
         --mail-user=${mailuser} --mail-type=${mailtype} --requeue --mem=${mem_Internal_Currency} \
		 --depend=afterok${depend_Refine_Extid_Merge} \
         "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Internal_Currency ${start_year} ${end_year} | awk '{print $NF}'`
echo "Submitted Internal_Currency Job: "${Internal_Currency_ID}
sleep 1

# Refine Cusip Merge
#### This step merges security-level data from the CUSIP Global Serices (CGS) master files into the holding details data.
depend_Refine_Cusip_Merge=""
if [ "$run_small" = true ] ; then
    Refine_Cusip_Merge_1_ID=`sbatch \
             --partition=${partition_Refine_Cusip_Merge_1} --time=${time_Refine_Cusip_Merge_1} \
             --nodes=${nodes} --ntasks=${ntasks_Refine_Cusip_Merge_1} --job-name=Refine_Cusip_Merge_1 \
             --output="${erroroutput_path}/Refine_Cusip_Merge_1-%A_%a.out" --error="${erroroutput_path}/Refine_Cusip_Merge_1-%A_%a.err" \
             --array=${array_small} \
             --mail-user=${mailuser} --mail-type=${mailtype} --requeue --mem=${mem_Refine_Cusip_Merge_1} \
             --depend=afterok:${Internal_Currency_ID} \
             "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Refine_Cusip_Merge ${start_year} ${end_year} | awk '{print $NF}'`
    echo "Submitted Refine_Cusip_Merge_1 Job: "${Refine_Cusip_Merge_1_ID}
    sleep 1

    depend_Refine_Cusip_Merge=":${Refine_Cusip_Merge_1_ID}"
fi
if [ "$run_large" = true ] ; then
    Refine_Cusip_Merge_2_ID=`sbatch \
             --partition=${partition_Refine_Cusip_Merge_2} --time=${time_Refine_Cusip_Merge_2} \
             --nodes=${nodes} --ntasks=${ntasks_Refine_Cusip_Merge_2} --job-name=Refine_Cusip_Merge_2 \
             --output="${erroroutput_path}/Refine_Cusip_Merge_2-%A_%a.out" --error="${erroroutput_path}/Refine_Cusip_Merge_2-%A_%a.err" \
             --array=${array_large} \
             --mail-user=${mailuser} --mail-type=${mailtype} --requeue --mem=${mem_Refine_Cusip_Merge_2} \
             --depend=afterok:${Internal_Currency_ID} \
             "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Refine_Cusip_Merge ${start_year} ${end_year} | awk '{print $NF}'`
    echo "Submitted Refine_Cusip_Merge_2 Job: "${Refine_Cusip_Merge_2_ID}
    sleep 1

    depend_Refine_Cusip_Merge="${depend_Refine_Cusip_Merge}:${Refine_Cusip_Merge_2_ID}"
fi

# Internal Class
#### This step finds the modal typecode assigned to each fund in the Morningstar data.
Internal_Class_ID=`sbatch \
         --partition=${partition_Internal_Class} --time=${time_Internal_Class} \
         --nodes=${nodes} --ntasks=${ntasks_Internal_Class} --job-name=Internal_Class \
		 --output="${erroroutput_path}/Internal_Class-%A_%a.out" --error="${erroroutput_path}/Internal_Class-%A_%a.err" \
		 --mail-user=${mailuser} --mail-type=${mailtype} --requeue --mem=${mem_Internal_Class} \
		 --depend=afterok${depend_Refine_Cusip_Merge} \
         "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Internal_Class ${start_year} ${end_year} | awk '{print $NF}'`
echo "Submitted Internal_Class Job: "${Internal_Class_ID}
sleep 1


# Manual Corrections
#### This step cleans several extraordinarily large positions in the holding details data.
Manual_Corrections_ID=`sbatch \
         --partition=${partition_Manual_Corrections} --time=${time_Manual_Corrections} \
         --nodes=${nodes} --ntasks=${ntasks_Manual_Corrections} --job-name=Manual_Corrections \
		 --output="${erroroutput_path}/Manual_Corrections-%A_%a.out" --error="${erroroutput_path}/Manual_Corrections-%A_%a.err" \
		 --mail-user=${mailuser} --mail-type=${mailtype} --requeue --mem=${mem_Manual_Corrections} \
         --depend=afterok${depend_Refine_Cusip_Merge} \
         "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Manual_Corrections ${start_year} ${end_year} | awk '{print $NF}'`
echo "Submitted Manual_Corrections Job: "${Manual_Corrections_ID}
sleep 1
#

############################################################################################################################
# SUBMIT JOBS TO CREATE MASTER FILES
############################################################################################################################

# job separator
;;&
"final" | "all" | "pre_unwind")

# set up dependcy if necessary 
# Set variable for whether to process small HoldingDetail files (those from 1986 to 2002)
if [ "$job" = "final" ] ; then
    depend=""
else
    depend="--depend=afterok:${Internal_Class_ID}:${Manual_Corrections_ID}"
fi

# Create Final Files
#### This step generates the final holding detail files that are used in the analysis.
depend_Create_Final_Files=""
if [ "$run_small" = true ] ; then
    Create_Final_Files_1_ID=`sbatch \
             --partition=${partition_Create_Final_Files_1} --time=${time_Create_Final_Files_1} \
             --nodes=${nodes} --ntasks=${ntasks_Create_Final_Files_1} --job-name=Create_Final_Files_1 \
             --output="${erroroutput_path}/Create_Final_Files_1-%A_%a.out" --error="${erroroutput_path}/Create_Final_Files_1-%A_%a.err" \
             --array=${array_small} \
             --mail-user=${mailuser} --mail-type=${mailtype} --requeue --mem=${mem_Create_Final_Files_1} \
             $depend \
             "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Create_Final_Files ${start_year} ${end_year} | awk '{print $NF}'`
    echo "Submitted Create_Final_Files_1 Job: "${Create_Final_Files_1_ID}
    sleep 1

    depend_Create_Final_Files=":${Create_Final_Files_1_ID}"
fi
if [ "$run_large" = true ] ; then
    Create_Final_Files_2_ID=`sbatch \
             --partition=${partition_Create_Final_Files_2} --time=${time_Create_Final_Files_2} \
             --nodes=${nodes} --ntasks=${ntasks_Create_Final_Files_2} --job-name=Create_Final_Files_2 \
             --output="${erroroutput_path}/Create_Final_Files_2-%A_%a.out" --error="${erroroutput_path}/Create_Final_Files_2-%A_%a.err" \
             --array=${array_large} \
             --mail-user=${mailuser} --mail-type=${mailtype} --requeue --mem=${mem_Create_Final_Files_2} \
             $depend \
             "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Create_Final_Files ${start_year} ${end_year} | awk '{print $NF}'`
    echo "Submitted Create_Final_Files_2 Job: "${Create_Final_Files_2_ID}
    sleep 1

    depend_Create_Final_Files="${depend_Create_Final_Files}:${Create_Final_Files_2_ID}"
fi


############################################################################################################################
# SUBMIT UNWIND STEPS
############################################################################################################################

# job separator
;;&
"mf_unwind" | "all")

# MF_Unwind
#### The 'Unwind_MF' steps above handle the unwinding of funds' positions in other funds.
#### This procedure is referred to as fund-in-fund unwinding.
# Unwind_MF_Prepare: Prepare some temporary files used by the fund-in-fund unwinding code
array_Unwind_MF_Prepare="$start_year-$end_year"
JOB_Unwind_MF_Prepare_ID=`sbatch \
         --partition=${partition_Unwind_MF_Prepare} --time=${time_Unwind_MF_Prepare} \
         --nodes=${nodes} --ntasks=${ntasks_Unwind_MF_Prepare} --job-name=Unwind_MF_Prepare \
         --output="${erroroutput_path}/Unwind_MF_Prepare-%A_%a.out" --error="${erroroutput_path}/Unwind_MF_Prepare-%A_%a.err" \
         --mail-user=${mailuser} --mail-type=${mailtype} --requeue \
		 --depend=afterok${depend_Create_Final_Files} \
         --mem=${mem_Unwind_MF_Prepare} --array=${array_Unwind_MF_Prepare} \
         "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Unwind_MF_Prepare ${start_year} ${end_year} | awk '{print $NF}'`
echo "Submitted Unwind_MF_Prepare Job: "${JOB_Unwind_MF_Prepare_ID}
sleep 1


# Unwind_MF_Positions_Unravel (Python): Handle the unraveling of positions of funds investing in other funds
array_Unwind_MF_Positions_Unravel="1-$n_halfs"
JOB_Unwind_MF_Positions_Unravel_ID=`sbatch \
         --partition=${partition_Unwind_MF_Positions_Unravel} --time=${time_Unwind_MF_Positions_Unravel} \
         --nodes=${nodes} --ntasks=${ntasks_Unwind_MF_Positions_Unravel} --job-name=Unwind_MF_Positions_Unravel \
         --output="${erroroutput_path}/Unwind_MF_Positions_Unravel-%A_%a.out" --error="${erroroutput_path}/Unwind_MF_Positions_Unravel-%A_%a.err" \
         --mail-user=${mailuser} --mail-type=${mailtype} --requeue \
		 --depend=afterok:${JOB_Unwind_MF_Prepare_ID} \
         --mem=${mem_Unwind_MF_Positions_Unravel} --array=${array_Unwind_MF_Positions_Unravel} \
         "${code_path}/morningstar/mf_unwind/Python_Controller.sh" ${U} Unwind_MF_Positions_Unravel | awk '{print $NF}'`
echo "Submitted Unwind_MF_Positions_Unravel Job: "${JOB_Unwind_MF_Positions_Unravel_ID}
sleep 1


# Unwind_MF_Positions_Consolidate: Consolidate adjustment factors used to compute appropriately re-scaled versions of
# the positions involved in the fund-in-fund unwinding
array_Unwind_MF_Positions_Consolidate="$start_year-$end_year"
JOB_Unwind_MF_Positions_Consolidate_ID=`sbatch \
         --partition=${partition_Unwind_MF_Positions_Consolidate} --time=${time_Unwind_MF_Positions_Consolidate} \
         --nodes=${nodes} --ntasks=${ntasks_Unwind_MF_Positions_Consolidate} --job-name=Unwind_MF_Positions_Consolidate \
         --output="${erroroutput_path}/Unwind_MF_Positions_Consolidate-%A_%a.out" --error="${erroroutput_path}/Unwind_MF_Positions_Consolidate-%A_%a.err" \
         --mail-user=${mailuser} --mail-type=${mailtype} --requeue \
         --depend=afterok:${JOB_Unwind_MF_Positions_Unravel_ID} \
         --mem=${mem_Unwind_MF_Positions_Consolidate} --array=${array_Unwind_MF_Positions_Consolidate} \
         "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Unwind_MF_Positions_Consolidate ${start_year} ${end_year} | awk '{print $NF}'`
echo "Submitted Unwind_MF_Positions_Consolidate Job: "${JOB_Unwind_MF_Positions_Consolidate_ID}
sleep 1


# Unwind_MF_Positions_Step2: Re-generate holding detail data reflecting the unraveling and rescaling of fund-in-fund positions
array_Unwind_MF_Positions_ReGenHD_1_Small="1-$n_NonUS1,$n_US0-$n_US1"
JOB_Unwind_MF_Positions_ReGenHD_1_Small_ID=`sbatch \
         --partition=${partition_Unwind_MF_Positions_ReGenHD_1_Small} --time=${time_Unwind_MF_Positions_ReGenHD_1_Small} \
         --nodes=${nodes} --ntasks=${ntasks_Unwind_MF_Positions_ReGenHD_1_Small} --job-name=Unwind_MF_Positions_ReGenHD_1_Small \
         --output="${erroroutput_path}/Unwind_MF_Positions_ReGenHD_1_Small-%A_%a.out" --error="${erroroutput_path}/Unwind_MF_Positions_ReGenHD_1_Small-%A_%a.err" \
         --mail-user=${mailuser} --mail-type=${mailtype} --requeue \
         --depend=afterok:${JOB_Unwind_MF_Positions_Consolidate_ID} \
         --mem=${mem_Unwind_MF_Positions_ReGenHD_1_Small} --array=${array_Unwind_MF_Positions_ReGenHD_1_Small} \
         "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Unwind_MF_Positions_ReGenHD_1 ${start_year} ${end_year} | awk '{print $NF}'`
echo "Submitted Unwind_MF_Positions_ReGenHD_1_Small Job: "${JOB_Unwind_MF_Positions_ReGenHD_1_Small_ID}
sleep 1

array_Unwind_MF_Positions_ReGenHD_1_Large="$n_NonUS2-$n_halfs,$n_US2-$n_US3"
JOB_Unwind_MF_Positions_ReGenHD_1_Large_ID=`sbatch \
         --partition=${partition_Unwind_MF_Positions_ReGenHD_1_Large} --time=${time_Unwind_MF_Positions_ReGenHD_1_Large} \
         --nodes=${nodes} --ntasks=${ntasks_Unwind_MF_Positions_ReGenHD_1_Large} --job-name=Unwind_MF_Positions_ReGenHD_1_Large \
         --output="${erroroutput_path}/Unwind_MF_Positions_ReGenHD_1_Large-%A_%a.out" --error="${erroroutput_path}/Unwind_MF_Positions_ReGenHD_1_Large-%A_%a.err" \
         --mail-user=${mailuser} --mail-type=${mailtype} --requeue \
         --depend=afterok:${JOB_Unwind_MF_Positions_Consolidate_ID} \
         --mem=${mem_Unwind_MF_Positions_ReGenHD_1_Large} --array=${array_Unwind_MF_Positions_ReGenHD_1_Large} \
         "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Unwind_MF_Positions_ReGenHD_1 ${start_year} ${end_year} | awk '{print $NF}'`
echo "Submitted Unwind_MF_Positions_ReGenHD_1_Large Job: "${JOB_Unwind_MF_Positions_ReGenHD_1_Large_ID}
sleep 1


# Unwind_MF_Positions_Step3: Re-generate holding detail data reflecting the unraveling and rescaling of fund-in-fund positions
array_Unwind_MF_Positions_ReGenHD_2_Small="1-$n_NonUS1,$n_US0-$n_US1"
JOB_Unwind_MF_Positions_ReGenHD_2_Small_ID=`sbatch \
         --partition=${partition_Unwind_MF_Positions_ReGenHD_2_Small} --time=${time_Unwind_MF_Positions_ReGenHD_2_Small} \
         --nodes=${nodes} --ntasks=${ntasks_Unwind_MF_Positions_ReGenHD_2_Small} --job-name=Unwind_MF_Positions_ReGenHD_2_Small \
         --output="${erroroutput_path}/Unwind_MF_Positions_ReGenHD_2_Small-%A_%a.out" --error="${erroroutput_path}/Unwind_MF_Positions_ReGenHD_2_Small-%A_%a.err" \
         --mail-user=${mailuser} --mail-type=${mailtype} --requeue \
         --depend=afterok:${JOB_Unwind_MF_Positions_ReGenHD_1_Small_ID}:${JOB_Unwind_MF_Positions_ReGenHD_1_Large_ID} \
         --mem=${mem_Unwind_MF_Positions_ReGenHD_2_Small} --array=${array_Unwind_MF_Positions_ReGenHD_2_Small} \
         "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Unwind_MF_Positions_ReGenHD_2 ${start_year} ${end_year} | awk '{print $NF}'`
echo "Submitted Unwind_MF_Positions_ReGenHD_2_Small Job: "${JOB_Unwind_MF_Positions_ReGenHD_2_Small_ID}
sleep 1

array_Unwind_MF_Positions_ReGenHD_2_Large="$n_NonUS2-$n_halfs,$n_US2-$n_US3"
JOB_Unwind_MF_Positions_ReGenHD_2_Large_ID=`sbatch \
         --partition=${partition_Unwind_MF_Positions_ReGenHD_2_Large} --time=${time_Unwind_MF_Positions_ReGenHD_2_Large} \
         --nodes=${nodes} --ntasks=${ntasks_Unwind_MF_Positions_ReGenHD_2_Large} --job-name=Unwind_MF_Positions_ReGenHD_2_Large \
         --output="${erroroutput_path}/Unwind_MF_Positions_ReGenHD_2_Large-%A_%a.out" --error="${erroroutput_path}/Unwind_MF_Positions_ReGenHD_2_Large-%A_%a.err" \
         --mail-user=${mailuser} --mail-type=${mailtype} --requeue \
         --depend=afterok:${JOB_Unwind_MF_Positions_ReGenHD_1_Small_ID}:${JOB_Unwind_MF_Positions_ReGenHD_1_Large_ID} \
         --mem=${mem_Unwind_MF_Positions_ReGenHD_2_Large} --array=${array_Unwind_MF_Positions_ReGenHD_2_Large} \
         "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Unwind_MF_Positions_ReGenHD_2 ${start_year} ${end_year} | awk '{print $NF}'`
echo "Submitted Unwind_MF_Positions_ReGenHD_2_Large Job: "${JOB_Unwind_MF_Positions_ReGenHD_2_Large_ID}
sleep 1


# Unwind_MF_Positions_Step4: Aggregate the fund-in-fund unraveled data to a yearly frequency
array_Unwind_MF_Positions_Aggregate="$start_year-$end_year"
JOB_Unwind_MF_Positions_Aggregate_ID=`sbatch \
         --partition=${partition_Unwind_MF_Positions_Aggregate} --time=${time_Unwind_MF_Positions_Aggregate} \
         --nodes=${nodes} --ntasks=${ntasks_Unwind_MF_Positions_Aggregate} --job-name=Unwind_MF_Positions_Aggregate \
         --output="${erroroutput_path}/Unwind_MF_Positions_Aggregate-%A_%a.out" --error="${erroroutput_path}/Unwind_MF_Positions_Aggregate-%A_%a.err" \
         --mail-user=${mailuser} --mail-type=${mailtype} --requeue \
         --depend=afterok:${JOB_Unwind_MF_Positions_ReGenHD_2_Small_ID}:${JOB_Unwind_MF_Positions_ReGenHD_2_Large_ID} \
         --mem=${mem_Unwind_MF_Positions_Aggregate} --array=${array_Unwind_MF_Positions_Aggregate} \
         "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Unwind_MF_Positions_Aggregate ${start_year} ${end_year} | awk '{print $NF}'`
echo "Submitted Unwind_MF_Positions_Aggregate Job: "${JOB_Unwind_MF_Positions_Aggregate_ID}
sleep 1

############################################################################################################################
# SUBMIT PANELIFY STEPS
############################################################################################################################

# job separator
;;&
"panelify" | "all")

# Reshape_Cut_Groups
JOB_Reshape_Cut_Groups_ID=`sbatch \
         --partition=${partition_Reshape_Cut_Groups} --time=${time_Reshape_Cut_Groups} \
         --nodes=${nodes} --ntasks=${ntasks_Reshape_Cut_Groups} --job-name=Reshape_Cut_Groups \
         --output="${erroroutput_path}/Reshape_Cut_Groups-%A_%a.out" --error="${erroroutput_path}/Reshape_Cut_Groups-%A_%a.err" \
         --mail-user=${mailuser} --mail-type=${mailtype} --requeue \
         --mem=${mem_Reshape_Cut_Groups} \
         --depend=afterok:${JOB_Unwind_MF_Positions_Aggregate_ID} \
           "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Reshape_Cut_Groups | awk '{print $NF}'`
echo "Submitted Reshape Cut Groups Job: "${JOB_Reshape_Cut_Groups_ID}
sleep 1


# Split_Annual_Files
array_Split_Annual_Files="1-15"
JOB_Split_Annual_Files_ID=`sbatch \
         --partition=${partition_Split_Annual_Files} --time=${time_Split_Annual_Files} \
         --nodes=${nodes} --ntasks=${ntasks_Split_Annual_Files} --job-name=Split_Annual_Files \
         --array=${array_Split_Annual_Files}  --output="${erroroutput_path}/Split_Annual_Files-%A_%a.out" --error="${erroroutput_path}/Split_Annual_Files-%A_%a.err" \
         --mail-user=${mailuser} --mail-type=${mailtype} --requeue \
         --depend=afterok:${JOB_Reshape_Cut_Groups_ID} --mem=${mem_Split_Annual_Files} \
           "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Split_Annual_Files | awk '{print $NF}'`
echo "Submitted Split Annual Files Job: "${JOB_Split_Annual_Files_ID}
sleep 1


# Reappend_Split_Annuals
array_Reappend_Split_Annuals="1-100"
JOB_Reappend_Split_Annuals_ID=`sbatch \
         --partition=${partition_Reappend_Split_Annuals} --time=${time_Reappend_Split_Annuals} \
         --nodes=${nodes} --ntasks=${ntasks_Reappend_Split_Annuals} --job-name=Reappend_Split_Annuals \
         --array=${array_Reappend_Split_Annuals}  --output="${erroroutput_path}/Reappend_Split_Annuals-%A_%a.out" --error="${erroroutput_path}/Reappend_Split_Annuals-%A_%a.err" \
         --mail-user=${mailuser} --mail-type=${mailtype} --requeue \
         --depend=afterok:${JOB_Split_Annual_Files_ID} --mem=${mem_Reappend_Split_Annuals} \
           "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Reappend_Split_Annuals | awk '{print $NF}'`
echo "Submitted Reappend_Split_Annuals Job: "${JOB_Reappend_Split_Annuals_ID}
sleep 1


Panelify_MPID_Splits_Dependency_ID=$JOB_Reappend_Split_Annuals_ID

# Panelify Splits
array_Panelify_MPID_Splits="1-100"
JOB_Panelify_MPID_Splits_ID=`sbatch \
         --partition=${partition_Panelify_MPID_Splits} --time=${time_Panelify_MPID_Splits} \
         --nodes=${nodes} --ntasks=${ntasks_Panelify_MPID_Splits} --job-name=Panelify_MPID_Splits \
         --array=${array_Panelify_MPID_Splits}  --output="${erroroutput_path}/Panelify_MPID_Splits-%A_%a.out" --error="${erroroutput_path}/Panelify_MPID_Splits-%A_%a.err" \
         --mail-user=${mailuser} --mail-type=${mailtype} --requeue \
         --mem=${mem_Panelify_MPID_Splits} --depend=afterok:${Panelify_MPID_Splits_Dependency_ID} \
           "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Panelify_MPID_Splits | awk '{print $NF}'`
echo "Submitted Panelify_MPID_Splits Job: "${JOB_Panelify_MPID_Splits_ID}
sleep 1

# Panelify Splits Covariates
array_Panelify_MPID_Splits_Covariates="1-100"
JOB_Panelify_MPID_Splits_Covariates_ID=`sbatch \
         --partition=${partition_Panelify_MPID_Splits_Covariates} --time=${time_Panelify_MPID_Splits_Covariates} \
         --nodes=${nodes} --ntasks=${ntasks_Panelify_MPID_Splits_Covariates} --job-name=Panelify_MPID_Splits_Covariates \
         --array=${array_Panelify_MPID_Splits_Covariates}  --output="${erroroutput_path}/Panelify_MPID_Splits_Covariates-%A_%a.out" --error="${erroroutput_path}/Panelify_MPID_Splits_Covariates-%A_%a.err" \
         --mail-user=${mailuser} --mail-type=${mailtype} --requeue \
         --mem=${mem_Panelify_MPID_Splits_Covariates} --depend=afterok:${JOB_Panelify_MPID_Splits_ID} \
           "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Panelify_MPID_Splits_Covariates | awk '{print $NF}'`
echo "Submitted Panelify_MPID_Splits_Covariates Job: "${JOB_Panelify_MPID_Splits_Covariates_ID}
sleep 1


Append_HD_Panels_Dependency_ID=$JOB_Panelify_MPID_Splits_Covariates_ID

# Append HD Panels
JOB_Append_HD_Panels_ID=`sbatch \
         --partition=${partition_Append_HD_Panels} --time=${time_Append_HD_Panels} \
         --nodes=${nodes} --ntasks=${ntasks_Append_HD_Panels} --job-name=Append_HD_Panels \
         --output="${erroroutput_path}/Append_HD_Panels-%A_%a.out" --error="${erroroutput_path}/Append_HD_Panels-%A_%a.err" \
         --mail-user=${mailuser} --mail-type=${mailtype} --requeue \
         --depend=afterok:${Append_HD_Panels_Dependency_ID} --mem=${mem_Append_HD_Panels} \
           "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Append_HD_Panels | awk '{print $NF}'`
echo "Submitted Append_HD_Panels Job: "${JOB_Append_HD_Panels_ID}
sleep 1


# Find Constant Panels
JOB_Find_Constant_Panels_ID=`sbatch \
         --partition=${partition_Find_Constant_Panels} --time=${time_Find_Constant_Panels} \
         --nodes=${nodes} --ntasks=${ntasks_Find_Constant_Panels} --job-name=Find_Constant_Panels \
         --output="${erroroutput_path}/Find_Constant_Panels-%A_%a.out" --error="${erroroutput_path}/Find_Constant_Panels-%A_%a.err" \
         --mail-user=${mailuser} --mail-type=${mailtype} --requeue \
         --depend=afterok:${JOB_Append_HD_Panels_ID} --mem=${mem_Find_Constant_Panels} \
           "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Find_Constant_Panels | awk '{print $NF}'`
echo "Submitted Find_Constant_Panels Job: "${JOB_Find_Constant_Panels_ID}
sleep 1


# Chain constant panels and collapse
JOB_Collapse_Panels_ID=`sbatch \
             --partition=${partition_Collapse_Panels} --time=${time_Collapse_Panels} \
             --nodes=${nodes} --ntasks=${ntasks_Collapse_Panels} --job-name=Collapse_Panels \
             --output="${erroroutput_path}/Collapse_Panels-%A_%a.out" --error="${erroroutput_path}/Collapse_Panels-%A_%a.err" \
             --mail-user=${mailuser} --mail-type=${mailtype} --requeue \
             --depend=afterok:${JOB_Find_Constant_Panels_ID} --mem=${mem_Collapse_Panels} \
               "${code_path}/morningstar/Morningstar_Build_Controller.sh" ${U} Collapse_Panels | awk '{print $NF}'`
echo "Submitted Collapse_Panels Job: "${JOB_Collapse_Panels_ID}
sleep 1


;;
esac

echo "All jobs submitted!"
